<!DOCTYPE html>
<html lang="en">
  <head>
  <meta charset="utf-8" />
  <meta http-equiv="X-UA-Compatible" content="IE=edge">
  <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0">


  <title>Prometheus报警AlertManager实战-blog.qikqiak.com|阳明的博客|Kubernetes|Docker|Python|Golang|Cloud Native</title>
  <meta property="og:title" content="Prometheus报警AlertManager实战" />
  <meta name="twitter:title" content="Prometheus报警AlertManager实战" />

  <meta name="description" content="在前面一文Kubernetes使用Prometheus搭建监控平台中我们知道了怎么使用Prometheus来搭建监控平台，也了解了grafana的使用。这篇文章就来说说报警系统的搭建，有人说报警用grafana就行了，实际上grafana对报警的支持真的很弱，而Prometheus提供的报警系统就强大很多，今天我们的主角就是AlertManager。">
  <meta property="og:description" content="在前面一文Kubernetes使用Prometheus搭建监控平台中我们知道了怎么使用Prometheus来搭建监控平台，也了解了grafana的使用。这篇文章就来说说报警系统的搭建，有人说报警用grafana就行了，实际上grafana对报警的支持真的很弱，而Prometheus提供的报警系统就强大很多，今天我们的主角就是AlertManager。">
  <meta name="twitter:description" content="在前面一文Kubernetes使用Prometheus搭建监控平台中我们知道了怎么使用Prometheus来搭建监控平台，也了解了grafana的使用。这篇文章就来说说报警系统的搭建，有人说报警用grafana就行了，实际上grafana对报警的支持真的很弱，而Prometheus提供的报警系统就强大很多，今天我们的主角就是AlertManager。">
  <meta name="author" content=""/>
  <link href='https://blog.qikqiak.com/img/favicon.ico' rel='icon' type='image/x-icon'/>
  <meta property="og:image" content="https://blog.qikqiak.com/img/avatar.jpeg" />
  <meta name="twitter:image" content="https://blog.qikqiak.com/img/avatar.jpeg" />
  <meta name="twitter:card" content="summary" />
  <meta property="og:url" content="https://blog.qikqiak.com/post/alertmanager-of-prometheus-in-practice/" />
  <meta property="og:type" content="website" />
  <meta property="og:site_name" content="阳明的博客" />

  <meta name="generator" content="Hugo 0.34" />
  <link rel="canonical" href="https://blog.qikqiak.com/post/alertmanager-of-prometheus-in-practice/" />
  <link rel="alternate" href="https://blog.qikqiak.com/index.xml" type="application/rss+xml" title="阳明的博客">
  <link href="https://cdn.bootcss.com/KaTeX/0.7.1/katex.min.css" rel="stylesheet">
  <link href="https://cdn.bootcss.com/font-awesome/4.7.0/css/font-awesome.min.css" rel="stylesheet">
  <link href="https://cdn.bootcss.com/bootstrap/3.3.7/css/bootstrap.min.css" rel="stylesheet">
  <link rel="stylesheet" href="https://blog.qikqiak.com/css/main.css?t=1235" />
  <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Lora:400,700,400italic,700italic" />
  <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Open+Sans:300italic,400italic,600italic,700italic,800italic,400,300,600,700,800" />
  <link rel="stylesheet" href="https://blog.qikqiak.com/css/pygment_highlights.css" />
  <link rel="stylesheet" href="https://blog.qikqiak.com/css/highlight.min.css" />
  <link rel="stylesheet" href="https://blog.qikqiak.com/css/prism.css?t=123" />
  <link rel="stylesheet" href="https://blog.qikqiak.com/css/search.css" />
<meta name="keywords" content="kubernetes, prometheus, alertmanager, 报警, 钉钉, webhook">
<script>
var _hmt = _hmt || [];
(function() {
  var hm = document.createElement("script");
  hm.src = "https://hm.baidu.com/hm.js?970de9920f6089c287a7fd366138d4e9";
  var s = document.getElementsByTagName("script")[0];
  s.parentNode.insertBefore(hm, s);
})();
</script>



<link href="https://cdn.bootcss.com/photoswipe/4.1.1/photoswipe.min.css" rel="stylesheet">
<link href="https://cdn.bootcss.com/photoswipe/4.1.1/default-skin/default-skin.min.css" rel="stylesheet">



<div class="pswp" tabindex="-1" role="dialog" aria-hidden="true">

<div class="pswp__bg"></div>

<div class="pswp__scroll-wrap">
    
    <div class="pswp__container">
      <div class="pswp__item"></div>
      <div class="pswp__item"></div>
      <div class="pswp__item"></div>
    </div>
    
    <div class="pswp__ui pswp__ui--hidden">
    <div class="pswp__top-bar">
      
      <div class="pswp__counter"></div>
      <button class="pswp__button pswp__button--close" title="Close (Esc)"></button>
      <button class="pswp__button pswp__button--share" title="Share"></button>
      <button class="pswp__button pswp__button--fs" title="Toggle fullscreen"></button>
      <button class="pswp__button pswp__button--zoom" title="Zoom in/out"></button>
      
      
      <div class="pswp__preloader">
        <div class="pswp__preloader__icn">
          <div class="pswp__preloader__cut">
            <div class="pswp__preloader__donut"></div>
          </div>
        </div>
      </div>
    </div>
    <div class="pswp__share-modal pswp__share-modal--hidden pswp__single-tap">
      <div class="pswp__share-tooltip"></div>
    </div>
    <button class="pswp__button pswp__button--arrow--left" title="Previous (arrow left)">
    </button>
    <button class="pswp__button pswp__button--arrow--right" title="Next (arrow right)">
    </button>
    <div class="pswp__caption">
      <div class="pswp__caption__center"></div>
    </div>
    </div>
    </div>
</div>

<script>
window.ga=window.ga||function(){(ga.q=ga.q||[]).push(arguments)};ga.l=+new Date;
ga('create', 'UA-69668147-3', 'auto');
ga('send', 'pageview');
</script>
<script async src='//www.google-analytics.com/analytics.js'></script>

</head>

  <body>
    <nav class="navbar navbar-default navbar-fixed-top navbar-custom">
  <div class="container-fluid">
    <div class="navbar-header">
      <button type="button" class="navbar-toggle" data-toggle="collapse" data-target="#main-navbar">
        <span class="sr-only">Toggle navigation</span>
        <span class="icon-bar"></span>
        <span class="icon-bar"></span>
        <span class="icon-bar"></span>
      </button>
      <a class="navbar-brand" href="https://blog.qikqiak.com/">阳明的博客</a>
    </div>

    <div class="collapse navbar-collapse" id="main-navbar">
      <ul class="nav navbar-nav navbar-right">
        
          
            <li>
              <a title="Home" href="https://ydzs.io">Home</a>
            </li>
          
        
          
            <li class="navlinks-container">
              <a class="navlinks-parent" href="javascript:void(0)">Categories</a>
              <div class="navlinks-children">
                
                  <a href="https://blog.qikqiak.com/page/archive/">Archive</a>
                
                  <a href="https://blog.qikqiak.com/tags">tags</a>
                
                  <a href="https://blog.qikqiak.com/tags/kubernetes">kubernetes</a>
                
                  <a href="https://blog.qikqiak.com/tags/python">python</a>
                
                  <a href="https://blog.qikqiak.com/tags/django">django</a>
                
                  <a href="https://blog.qikqiak.com/tags/ops">devops</a>
                
              </div>
            </li>
          
        
          
            <li class="navlinks-container">
              <a class="navlinks-parent" href="javascript:void(0)">海马学院</a>
              <div class="navlinks-children">
                
                  <a href="https://www.haimaxy.com/course/6y02wp/?utm_source=blog&amp;utm_campaign=referral&amp;utm_medium=topmenu">Python微服务实战</a>
                
                  <a href="https://www.haimaxy.com/course/pww13p/?utm_source=blog&amp;utm_campaign=referral&amp;utm_medium=topmenu">Webpack3.x 入门</a>
                
                  <a href="https://www.haimaxy.com/course/pjrqxm/?utm_source=blog&amp;utm_campaign=referral&amp;utm_medium=topmenu">Kubernetes 集群搭建</a>
                
              </div>
            </li>
          
        
          
            <li>
              <a title="About" href="https://blog.qikqiak.com/page/about/">About</a>
            </li>
          
        

        

        

        
          <li>
            <a href="#modalSearch" data-toggle="modal" data-target="#modalSearch" style="outline: none;">
              <span class="hidden-sm hidden-md hidden-lg">Search</span> <span id="searchGlyph" class="glyphicon glyphicon-search"></span>
            </a>
          </li>
          

      </ul>
    </div>

    <div class="avatar-container">
      <div class="avatar-img-border">
        
          <a title="阳明的博客" href="https://blog.qikqiak.com/">
            <img class="avatar-img" src="https://blog.qikqiak.com/img/avatar.jpeg" alt="阳明的博客" />
          </a>
        
      </div>
    </div>

  </div>
</nav>





  <div id="modalSearch" class="modal fade" role="dialog">
    <div class="modal-dialog">
      <div class="modal-content">
        <div class="modal-header">
          <button type="button" class="close" data-dismiss="modal">&times;</button>
          <h4 class="modal-title">Search blog.qikqiak.com</h4>
        </div>
        <div class="modal-body">
            
<div class="aa-input-container" id="aa-input-container">
    <input type="search" id="aa-search-input" class="aa-input-search" placeholder="Search for titles or URIs..." name="search" autocomplete="off" />
    <svg class="aa-input-icon" viewBox="654 -372 1664 1664">
        <path d="M1806,332c0-123.3-43.8-228.8-131.5-316.5C1586.8-72.2,1481.3-116,1358-116s-228.8,43.8-316.5,131.5  C953.8,103.2,910,208.7,910,332s43.8,228.8,131.5,316.5C1129.2,736.2,1234.7,780,1358,780s228.8-43.8,316.5-131.5  C1762.2,560.8,1806,455.3,1806,332z M2318,1164c0,34.7-12.7,64.7-38,90s-55.3,38-90,38c-36,0-66-12.7-90-38l-343-342  c-119.3,82.7-252.3,124-399,124c-95.3,0-186.5-18.5-273.5-55.5s-162-87-225-150s-113-138-150-225S654,427.3,654,332  s18.5-186.5,55.5-273.5s87-162,150-225s138-113,225-150S1262.7-372,1358-372s186.5,18.5,273.5,55.5s162,87,225,150s113,138,150,225  S2062,236.7,2062,332c0,146.7-41.3,279.7-124,399l343,343C2305.7,1098.7,2318,1128.7,2318,1164z" />
    </svg>
</div>
<script src="https://res.cloudinary.com/jimmysong/raw/upload/rootsongjc-hugo/algoliasearch.min.js"></script>
<script src="https://res.cloudinary.com/jimmysong/raw/upload/rootsongjc-hugo/autocomplete.min.js"></script>

<script>
var client = algoliasearch("1JDRAS0AZR", "8804ac109158bb3bb60d74ce98fa332f");
var index = client.initIndex('prod_blog');

autocomplete('#aa-search-input',
{ hint: false}, {
    source: autocomplete.sources.hits(index, {hitsPerPage: 5}),
    
    displayKey: 'name',
    
    templates: {
        
        suggestion: function(suggestion) {
            return '<span>' + '<a href="https://blog.qikqiak.com/post/' + suggestion.slug + '">' +
            suggestion._highlightResult.title.value + '</a></span>';
        }
    }
});
</script>
        </div>
        <div class="modal-footer">
          <button type="button" class="btn btn-default" data-dismiss="modal">close</button>
        </div>
      </div>
    </div>
  </div>

    
  
  
  




  
    <div id="header-big-imgs" data-num-img=1 data-img-src-1="https://blog.qikqiak.com/img/posts/photo-1512389142860-9c449e58a543.jpeg" ></div>
  

  <header class="header-section has-img">
    
      <div class="intro-header big-img">
        
        <div class="container">
          <div class="row">
            <div class="col-lg-8 col-lg-offset-2 col-md-10 col-md-offset-1">
              <div class="post-heading">
                <h1>Prometheus报警AlertManager实战</h1>
                  
                  
                    <span class="post-meta">
  Posted on December 27, 2017
  
</span>


                  
              </div>
            </div>
          </div>
        </div>
        <span class="img-desc" style="display: inline;"></span>
      </div>
    
    <div class="intro-header no-img">
      
      <div class="container">
        <div class="row">
          <div class="col-lg-8 col-lg-offset-2 col-md-10 col-md-offset-1">
            <div class="post-heading">
              <h1>Prometheus报警AlertManager实战</h1>
                
                
                  <span class="post-meta">
  Posted on December 27, 2017
  
</span>


                
            </div>
          </div>
        </div>
      </div>
    </div>
  </header>


    
<div class="container" role="main">
  <div class="row">
    <div class="col-lg-8 col-lg-offset-2 col-md-10 col-md-offset-1">

      <div>
          
          
          <h5 id="tags" style="margin-top: 0px;">标签:
            
                <a href="https://blog.qikqiak.com/tags/kubernetes/">kubernetes</a> &nbsp;
            
                <a href="https://blog.qikqiak.com/tags/prometheus/">prometheus</a> &nbsp;
            
                <a href="https://blog.qikqiak.com/tags/alertmanager/">alertmanager</a> &nbsp;
            
          </h5>
          
      </div>

      <article role="main" class="blog-post">
        <p>在前面一文<a href="https://blog.qikqiak.com/post/kubernetes-monitor-prometheus-grafana/">Kubernetes使用Prometheus搭建监控平台</a>中我们知道了怎么使用<code>Prometheus</code>来搭建监控平台，也了解了<code>grafana</code>的使用。这篇文章就来说说报警系统的搭建，有人说报警用<code>grafana</code>就行了，实际上<code>grafana</code>对报警的支持真的很弱，而<code>Prometheus</code>提供的报警系统就强大很多，今天我们的主角就是<code>AlertManager</code>。</p>

<p>
欢迎大家加入我们的知识星球：<code>Kubernetes</code>。
<img src="https://blog.qikqiak.com/img/xq.png" alt="知识星球" /></p>

<p>扫描下面的二维码(或微信搜索<code>k8s技术圈</code>)关注我们的微信公众帐号，在微信公众帐号中回复 <strong>加群</strong> 即可加入到我们的 kubernetes 讨论群里面共同学习。
<img src="https://blog.qikqiak.com/img/posts/qrcode_for_gh_d6dd87b6ceb4_430.jpg" alt="qrcode" /></p>

<h2 id="alertmanager-简介">AlertManager 简介</h2>

<p><code>Prometheus</code>将数据采集和报警分成了两个模块。报警规则配置在<code>Prometheus Servers</code>上，然后发送报警信息到<code>AlertManger</code>，然后我们的<code>AlertManager</code>就来管理这些报警信息，包括<strong>silencing</strong>、<strong>inhibition</strong>，聚合报警信息过后通过email、PagerDuty、HipChat、Slack 等方式发送消息提示。</p>

<p>让<code>AlertManager</code>提供服务总的来说就下面3步：
* 安装和配置<code>AlertManger</code>
* 配置<code>Prometheus</code>来和<code>AlertManager</code>通信
* 在<code>Prometheus</code>中创建报警规则</p>

<h2 id="安装和配置-alertmanager">安装和配置<code>AlertManager</code></h2>

<p>从官方文档<a href="https://prometheus.io/docs/alerting/configuration/">https://prometheus.io/docs/alerting/configuration/</a>中我们可以看到下载<code>alertmanager</code>二进制文件过后，通过下面的命令就可运行</p>

<pre><code class="language-shell">$ ./alertmanager -config.file=simple.yml
</code></pre>

<p>可以通过<code>-config.file</code>来指定相应的配置文件，我们这里在<code>kubernetes</code>平台上面来运行我们的<code>AlertManager</code>，所以我们直接用<code>docker</code>镜像的方式来安装，使用的镜像是：<code>quay.io/prometheus/alertmanager:v0.12.0</code>，如果你因为某些原因不能拉取该镜像，可以直接去下载<code>AlertManager</code>的<a href="https://github.com/prometheus/alertmanager">源代码</a>，然后自己构建打包一个<code>docker</code>镜像也是一样的。</p>

<p>通过<code>ConfigMap</code>来指定配置文件，如下:</p>

<pre><code class="language-yaml">kind: ConfigMap
apiVersion: v1
metadata:
  name: alertmanager
  namespace: kube-ops
data:
  config.yml: |-
    global:
      resolve_timeout: 5m
    route:
      receiver: webhook
      group_wait: 30s
      group_interval: 5m
      repeat_interval: 4h
      group_by: [alertname]
      routes:
      - receiver: webhook
        group_wait: 10s
        match:
          team: node
    receivers:
    - name: webhook
      webhook_configs:
      - url: 'http://apollo/hooks/dingtalk/'
        send_resolved: true
      - url: 'http://apollo/hooks/prome/'
        send_resolved: true
</code></pre>

<p>我们这里定义了两个<code>webhook</code>，其中一个是钉钉，另外一个是email。关于钉钉机器人可以查看<a href="https://open-doc.dingtalk.com/docs/doc.htm?spm=a219a.7629140.0.0.6LZJ6G&amp;treeId=257&amp;articleId=105735&amp;docType=1">官方文档</a>，钉钉的报警通知很简单，在你们的钉钉群里面添加一个机器人然后就可以获取到一个带<code>token</code>的URL，然后根据上面的文档往这个URL发送数据即可。</p>

<h2 id="配置-prometheus-来和-alertmanager-通信">配置<code>Prometheus</code>来和<code>AlertManager</code>通信</h2>

<p>在我们之前的<code>Prometheus</code>的<code>ConfigMap</code>中增加<code>AlertManager</code>的配置：</p>

<pre><code class="language-yaml">alerting:
  alertmanagers:
    - static_configs:
      - targets: [&quot;localhost:9093&quot;]
</code></pre>

<p>由于我们这个地方将<code>Prometheus</code>和<code>AlertManager</code>部署在同一个<code>POD</code>中，而<code>AlertManager</code>的默认端口是9093，所以直接使用<code>localhost:9093</code>就可以互相通信了。
<img src="https://blog.qikqiak.com/img/posts/prometheus-alertmanger-pod.jpg" alt="prometheus-pod" /></p>

<h2 id="在-prometheus-中创建报警规则">在<code>Prometheus</code>中创建报警规则</h2>

<p>同样的我们还需要添加报警规则：</p>

<pre><code class="language-yaml">rule_files:
    - /etc/prometheus/rules.yml
</code></pre>

<p>其中<code>rule_files</code>就是用来指定报警规则的，这里我们将<code>rules.yml</code>用<code>ConfigMap</code>的形式挂载到<code>/etc/prometheus</code>目录下面即可:</p>

<pre><code class="language-yaml">rules.yml: |
    groups:
    - name: test-rule
      rules:
      - alert: NodeFilesystemUsage
        expr: (node_filesystem_size{device=&quot;rootfs&quot;} - node_filesystem_free{device=&quot;rootfs&quot;}) / node_filesystem_size{device=&quot;rootfs&quot;} * 100 &gt; 80
        for: 2m
        labels:
          team: node
        annotations:
          summary: &quot;{{$labels.instance}}: High Filesystem usage detected&quot;
          description: &quot;{{$labels.instance}}: Filesystem usage is above 80% (current value is: {{ $value }}&quot;
      - alert: NodeMemoryUsage
        expr: (node_memory_MemTotal - (node_memory_MemFree+node_memory_Buffers+node_memory_Cached )) / node_memory_MemTotal * 100 &gt; 80
        for: 2m
        labels:
          team: node
        annotations:
          summary: &quot;{{$labels.instance}}: High Memory usage detected&quot;
          description: &quot;{{$labels.instance}}: Memory usage is above 80% (current value is: {{ $value }}&quot;
      - alert: NodeCPUUsage
        expr: (100 - (avg by (instance) (irate(node_cpu{job=&quot;kubernetes-node-exporter&quot;,mode=&quot;idle&quot;}[5m])) * 100)) &gt; 80
        for: 2m
        labels:
          team: node
        annotations:
          summary: &quot;{{$labels.instance}}: High CPU usage detected&quot;
          description: &quot;{{$labels.instance}}: CPU usage is above 80% (current value is: {{ $value }}&quot;
</code></pre>

<p>我们这里添加了3条测试的报警规则，分别是节点的文件系统、节点内存和CPU的使用量，如果大于了80%的话就触发label为<code>team=node</code>的<code>receiver</code>(上面alertmanager 配置文件中)，我们可以看到上面的配置就会匹配<code>webhook</code>这个<code>receiver</code>，然后会将报警信息<strong>POST</strong>到我们提供的两个HOOK 接口中(一个是通知钉钉的，另外一个是发邮件的)。</p>

<p>一个报警信息在生命周期内有下面3中状态：</p>

<ul>
<li>inactive: 表示当前报警信息既不是<code>firing</code>状态也不是<code>pending</code>状态</li>
<li>pending: 表示在设置的阈值时间范围内被激活了</li>
<li>firing: 表示超过设置的阈值时间被激活了</li>
</ul>

<p><img src="https://blog.qikqiak.com/img/posts/promethus-alert-module.png" alt="prometheus-alert-module" /></p>

<p>我们直接点击上面的<code>expr</code>会直接跳转到<code>graph</code>页面查询，我们在制定报警规则的时候同样可以先在<code>Prometheus</code>里面先测试我们的表达式:
<img src="https://blog.qikqiak.com/img/posts/alert-info.png" alt="alert-info" /></p>

<h2 id="全部配置">全部配置</h2>

<p>整体的<code>ConfigMap</code>配置文件如下<code>cm.yaml</code>:</p>

<pre><code class="language-yaml">apiVersion: v1
kind: ConfigMap
metadata:
  name: prometheus-config
  namespace: kube-ops
data:
  prometheus.yml: |
    global:
      scrape_interval: 30s
      scrape_timeout: 30s

    rule_files:
    - /etc/prometheus/rules.yml

    alerting:
      alertmanagers:
        - static_configs:
          - targets: [&quot;localhost:9093&quot;]

    scrape_configs:
    - job_name: 'prometheus'
      static_configs:
        - targets: ['localhost:9090']

    - job_name: 'kubernetes-apiservers'
      kubernetes_sd_configs:
      - role: endpoints
      scheme: https
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
      relabel_configs:
      - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
        action: keep
        regex: default;kubernetes;https

    - job_name: 'kubernetes-nodes'
      scheme: https
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
      kubernetes_sd_configs:
      - role: node
      relabel_configs:
      - action: labelmap
        regex: __meta_kubernetes_node_label_(.+)
      - target_label: __address__
        replacement: kubernetes.default.svc:443
      - source_labels: [__meta_kubernetes_node_name]
        regex: (.+)
        target_label: __metrics_path__
        replacement: /api/v1/nodes/${1}/proxy/metrics

    - job_name: 'kubernetes-cadvisor'
      scheme: https
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
      kubernetes_sd_configs:
      - role: node
      relabel_configs:
      - action: labelmap
        regex: __meta_kubernetes_node_label_(.+)
      - target_label: __address__
        replacement: kubernetes.default.svc:443
      - source_labels: [__meta_kubernetes_node_name]
        regex: (.+)
        target_label: __metrics_path__
        replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor

    - job_name: 'kubernetes-node-exporter'
      scheme: http
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
      kubernetes_sd_configs:
      - role: node
      relabel_configs:
      - action: labelmap
        regex: __meta_kubernetes_node_label_(.+)
      - source_labels: [__meta_kubernetes_role]
        action: replace
        target_label: kubernetes_role
      - source_labels: [__address__]
        regex: '(.*):10250'
        replacement: '${1}:31672'
        target_label: __address__

    - job_name: 'kubernetes-service-endpoints'
      kubernetes_sd_configs:
      - role: endpoints
      relabel_configs:
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
        action: keep
        regex: true
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
        action: replace
        target_label: __scheme__
        regex: (https?)
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
        action: replace
        target_label: __metrics_path__
        regex: (.+)
      - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
        action: replace
        target_label: __address__
        regex: ([^:]+)(?::\d+)?;(\d+)
        replacement: $1:$2
      - action: labelmap
        regex: __meta_kubernetes_service_label_(.+)
      - source_labels: [__meta_kubernetes_namespace]
        action: replace
        target_label: kubernetes_namespace
      - source_labels: [__meta_kubernetes_service_name]
        action: replace
        target_label: kubernetes_name

    - job_name: 'kubernetes-services'
      metrics_path: /probe
      params:
        module: [http_2xx]

      kubernetes_sd_configs:
      - role: service

      relabel_configs:
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_probe]
        action: keep
        regex: true
      - source_labels: [__address__]
        target_label: __param_target
      - target_label: __address__
        replacement: blackbox-exporter.example.com:9115
      - source_labels: [__param_target]
        target_label: instance
      - action: labelmap
        regex: __meta_kubernetes_service_label_(.+)
      - source_labels: [__meta_kubernetes_namespace]
        target_label: kubernetes_namespace
      - source_labels: [__meta_kubernetes_service_name]
        target_label: kubernetes_name

  rules.yml: |
    groups:
    - name: test-rule
      rules:
      - alert: NodeFilesystemUsage
        expr: (node_filesystem_size{device=&quot;rootfs&quot;} - node_filesystem_free{device=&quot;rootfs&quot;}) / node_filesystem_size{device=&quot;rootfs&quot;} * 100 &gt; 80
        for: 2m
        labels:
          team: node
        annotations:
          summary: &quot;{{$labels.instance}}: High Filesystem usage detected&quot;
          description: &quot;{{$labels.instance}}: Filesystem usage is above 80% (current value is: {{ $value }}&quot;
      - alert: NodeMemoryUsage
        expr: (node_memory_MemTotal - (node_memory_MemFree+node_memory_Buffers+node_memory_Cached )) / node_memory_MemTotal * 100 &gt; 80
        for: 2m
        labels:
          team: node
        annotations:
          summary: &quot;{{$labels.instance}}: High Memory usage detected&quot;
          description: &quot;{{$labels.instance}}: Memory usage is above 80% (current value is: {{ $value }}&quot;
      - alert: NodeCPUUsage
        expr: (100 - (avg by (instance) (irate(node_cpu{job=&quot;kubernetes-node-exporter&quot;,mode=&quot;idle&quot;}[5m])) * 100)) &gt; 80
        for: 2m
        labels:
          team: node
        annotations:
          summary: &quot;{{$labels.instance}}: High CPU usage detected&quot;
          description: &quot;{{$labels.instance}}: CPU usage is above 80% (current value is: {{ $value }}&quot;

---
kind: ConfigMap
apiVersion: v1
metadata:
  name: alertmanager
  namespace: kube-ops
data:
  config.yml: |-
    global:
      resolve_timeout: 5m
    route:
      receiver: webhook
      group_wait: 30s
      group_interval: 5m
      repeat_interval: 4h
      group_by: [alertname]
      routes:
      - receiver: webhook
        group_wait: 10s
        match:
          team: node
    receivers:
    - name: webhook
      webhook_configs:
      - url: 'http://apollo/hooks/dingtalk/'
        send_resolved: true
      - url: 'http://apollo/hooks/prome/'
        send_resolved: true
</code></pre>

<p>执行下面的命令即可：</p>

<pre><code class="language-shell">$ kubectl apply -f cm.yaml
</code></pre>

<p>然后部署的文件<code>deploy.yaml</code>如下：</p>

<pre><code class="language-yaml">kind: Deployment
metadata:
  labels:
    k8s-app: prometheus
  name: prometheus
  namespace: kube-ops
spec:
  replicas: 1
  template:
    metadata:
      labels:
        k8s-app: prometheus
    spec:
      serviceAccountName: prometheus
      containers:
      - image: prom/prometheus:v2.0.0-rc.3
        name: prometheus
        command:
        - &quot;/bin/prometheus&quot;
        args:
        - &quot;--config.file=/etc/prometheus/prometheus.yml&quot;
        - &quot;--storage.tsdb.path=/prometheus&quot;
        - &quot;--storage.tsdb.retention=24h&quot;
        ports:
        - containerPort: 9090
          protocol: TCP
          name: http
        volumeMounts:
        - mountPath: &quot;/prometheus&quot;
          name: data
          subPath: prometheus/data
        - mountPath: &quot;/etc/prometheus&quot;
          name: config-volume
        resources:
          requests:
            cpu: 100m
            memory: 100Mi
          limits:
            cpu: 200m
            memory: 1Gi
      - image: quay.io/prometheus/alertmanager:v0.12.0
        name: alertmanager
        args:
        - &quot;-config.file=/etc/alertmanager/config.yml&quot;
        - &quot;-storage.path=/alertmanager&quot;
        ports:
        - containerPort: 9093
          protocol: TCP
          name: http
        volumeMounts:
        - name: alertmanager-config-volume
          mountPath: /etc/alertmanager
        resources:
          requests:
            cpu: 50m
            memory: 50Mi
          limits:
            cpu: 200m
            memory: 200Mi
      volumes:
      - name: data
        emptyDir: {}
      - configMap:
          name: prometheus-config
        name: config-volume
      - name: alertmanager-config-volume
        configMap:
          name: alertmanager
</code></pre>

<p>执行下面的命令即可：
执行下面的命令即可：</p>

<pre><code class="language-shell">$ kubectl apply -f deploy.yaml
</code></pre>

<p>最后来看一张我们钉钉收到的报警信息吧:
<img src="https://blog.qikqiak.com/img/posts/prometheus-alertmanager-dingtalk.png" alt="dingtalk" /></p>

<blockquote>
<p>注意：本文章的实现都是针对<code>Prometheus2.x</code>版本，1.x 版本配置会有所不同</p>
</blockquote>

<p>本文中用到的<code>kubernetes</code>相关的<code>yaml</code>文件我已经归档到<code>github</code>上，可以在这里<a href="https://github.com/cnych/k8s-repo/tree/master/prometheus">https://github.com/cnych/k8s-repo/tree/master/prometheus</a>查看</p>

<h2 id="参考文档">参考文档</h2>

<ul>
<li><a href="https://prometheus.io/docs/alerting/configuration/">https://prometheus.io/docs/alerting/configuration/</a></li>
<li><a href="https://pracucci.com/prometheus-understanding-the-delays-on-alerting.html">Prometheus: understanding the delays on alerting</a></li>
</ul>

<p>欢迎大家加入我们的知识星球：<code>Kubernetes</code>。
<img src="https://blog.qikqiak.com/img/xq.png" alt="知识星球" /></p>

<p>扫描下面的二维码(或微信搜索<code>k8s技术圈</code>)关注我们的微信公众帐号，在微信公众帐号中回复 <strong>加群</strong> 即可加入到我们的 kubernetes 讨论群里面共同学习。
<img src="https://blog.qikqiak.com/img/posts/qrcode_for_gh_d6dd87b6ceb4_430.jpg" alt="qrcode" /></p>
      </article>

      
        

<h3>相关文章</h3>
<ul style="margin-bottom: 25px;">
    
    <li><a href="https://blog.qikqiak.com/post/kubernetes-monitor-prometheus-grafana/">Kubernetes使用Prometheus搭建监控平台</a></li>
    
    <li><a href="https://blog.qikqiak.com/post/update-prometheus-2-in-kubernetes/">Kubernetes 下升级Prometheus2.0</a></li>
    
    <li><a href="https://blog.qikqiak.com/post/kubernetes-logs-collect/">kubernetes 日志收集方案</a></li>
    
    <li><a href="https://blog.qikqiak.com/post/socketio-multiple-nodes-in-kubernetes/">kubernetes 下实现socket.io 的集群模式</a></li>
    
    <li><a href="https://blog.qikqiak.com/post/install-docker-registry-harbor-in-kubernetes/">在kubernetes 集群上搭建docker 私有仓库Harbor</a></li>
    
    <li><a href="https://blog.qikqiak.com/post/update-kubernetes-dashboard-more-secure/">kubernetes dashboard 升级之路</a></li>
    
    <li><a href="https://blog.qikqiak.com/post/manual-install-high-available-kubernetes-cluster/">手动搭建高可用的kubernetes 集群</a></li>
    
    <li><a href="https://blog.qikqiak.com/post/kubernetes-rollout-update/">Kubernetes Deployment滚动升级</a></li>
    
    <li><a href="https://blog.qikqiak.com/post/first-use-helm-on-kubernetes/">Kubernetes Helm 初体验</a></li>
    
    <li><a href="https://blog.qikqiak.com/post/understand-kubernetes-configmap-and-secrets/">kubernetes ConfigMap 和 Secrets</a></li>
    
</ul>

      

      
      <ul class="pager blog-pager">
        
          <li class="previous">
            <a href="https://blog.qikqiak.com/post/webpack3-get-started/" data-toggle="tooltip" data-placement="top" title="webpack3入门到放肆视频教程">&larr; Previous Post</a>
          </li>
        
        
          <li class="next">
            <a href="https://blog.qikqiak.com/post/tdd-develop-python-microservice-app/" data-toggle="tooltip" data-placement="top" title="TDD开发容器化的Python微服务应用(一)">Next Post &rarr;</a>
          </li>
        
      </ul>
      

      

      
      <div id="git-comments"></div>
      <link rel="stylesheet" href="https://imsun.github.io/gitment/style/default.css">
      <script src="https://imsun.github.io/gitment/dist/gitment.browser.js"></script>
      <script>
      var gitment = new Gitment({
        id: 'alertmanager-of-prometheus-in-practice',
        owner: 'cnych',
        repo: 'blog',
        oauth: {
          client_id: 'bdb76dbb2e9d0786e350',
          client_secret: 'b454b2a08013fd0e32013be7a63fa8fcb262b6c4',
        }
      })
      gitment.render('git-comments')
      </script>
      

    </div>
  </div>
</div>

    <footer>
  <div class="container">
    <div class="row">
      <div class="col-lg-8 col-lg-offset-2 col-md-10 col-md-offset-1">
        <ul class="list-inline text-center footer-links">
          
              <li>
                <a href="mailto:icnych@gmail.com" title="Email me">
                  <span class="fa-stack fa-lg">
                    <i class="fa fa-circle fa-stack-2x"></i>
                    <i class="fa fa-envelope fa-stack-1x fa-inverse"></i>
                  </span>
                </a>
              </li>
          
              <li>
                <a href="https://github.com/cnych" title="GitHub">
                  <span class="fa-stack fa-lg">
                    <i class="fa fa-circle fa-stack-2x"></i>
                    <i class="fa fa-github fa-stack-1x fa-inverse"></i>
                  </span>
                </a>
              </li>
          
              <li>
                <a href="https://weibo.com/cnych" title="微博">
                  <span class="fa-stack fa-lg">
                    <i class="fa fa-circle fa-stack-2x"></i>
                    <i class="fa fa-weibo fa-stack-1x fa-inverse"></i>
                  </span>
                </a>
              </li>
          
              <li>
                <a href="https://instagram.com/cnych" title="Instagram">
                  <span class="fa-stack fa-lg">
                    <i class="fa fa-circle fa-stack-2x"></i>
                    <i class="fa fa-instagram fa-stack-1x fa-inverse"></i>
                  </span>
                </a>
              </li>
          
          
          <li>
            <a href="https://blog.qikqiak.com/index.xml" title="RSS">
              <span class="fa-stack fa-lg">
                <i class="fa fa-circle fa-stack-2x"></i>
                <i class="fa fa-rss fa-stack-1x fa-inverse"></i>
              </span>
            </a>
          </li>
          
        </ul>
        <p class="credits copyright text-muted">
          

          &nbsp;&bull;&nbsp;
          2018

          
            &nbsp;&bull;&nbsp;
            <a href="https://blog.qikqiak.com/">阳明的博客</a>
          
        </p>
        
        <p class="credits theme-by text-muted">
          <a href="http://gohugo.io">Hugo v0.34</a> powered &nbsp;&bull;&nbsp; Theme by <a href="http://deanattali.com/beautiful-jekyll/">Beautiful Jekyll</a> adapted to <a href="https://github.com/halogenica/beautifulhugo">Beautiful Hugo</a>
          
        </p>
      </div>
    </div>
  </div>
</footer>

<script src="https://cdn.bootcss.com/KaTeX/0.7.1/katex.min.js"></script>
<script src="https://cdn.bootcss.com/KaTeX/0.7.1/contrib/auto-render.min.js"></script>
<script src="https://cdn.bootcss.com/jquery/1.12.4/jquery.min.js"></script>
<script src="https://cdn.bootcss.com/bootstrap/3.3.7/js/bootstrap.min.js"></script>
<script src="https://blog.qikqiak.com/js/main.min.js"></script>
<script src="https://blog.qikqiak.com/js/highlight.min.js"></script>
<script> hljs.initHighlightingOnLoad(); </script>
<script> renderMathInElement(document.body); </script>
<script src="https://blog.qikqiak.com/js/prism.js?t=123"></script>
<script src="https://cdn.bootcss.com/photoswipe/4.1.1/photoswipe.min.js"></script>
<script src="https://cdn.bootcss.com/photoswipe/4.1.1/photoswipe-ui-default.min.js"></script>
<script src="https://blog.qikqiak.com/js/load-photoswipe.min.js"></script>



<script async src="https://www.googletagmanager.com/gtag/js?id=UA-69668147-3"></script>
<script>
  window.dataLayer = window.dataLayer || [];
  function gtag(){dataLayer.push(arguments);}
  gtag('js', new Date());
  gtag('config', 'UA-69668147-3');
</script>

  </body>
</html>

